package main;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Spider {
	private static int MAX_PAGE_COUNT = 5000;
    private long startTime;
	private HashSet<String> visited;
	private Queue<PageInfo> my_PageBuffer; 
	private Queue<String> my_PageToRetrieveBuffer; 
	private String[] my_SeedList = {"intelligence","artificial","agent", "university","research", "science", "robot"};
	private String my_BaseURL;
	private Report my_report;
	private int totalURL;
	
	public Spider(String the_url){//, List<String> the_seedlist){
		my_BaseURL = the_url;
		
		visited = new HashSet<String>();
		my_PageBuffer = new LinkedList<PageInfo>();
		my_PageToRetrieveBuffer = new LinkedList<String>();
		startTime = System.nanoTime();
		totalURL = 0;
	//	my_SeedList = the_seedlist;
		
	}
	private void scanPage(){
		
		while (my_PageToRetrieveBuffer.isEmpty()==false) {//MAX_PAGE_COUNT) {
			//if (my_PageToRetrieveBuffer.isEmpty()==false){
			List<String>my_LinkList = new ArrayList<String>();	
			String var_currentLink = my_PageToRetrieveBuffer.poll();
				Retriever my_retriever = new Retriever(var_currentLink);
				
					my_LinkList = my_retriever.getLinks();
				
				totalURL = my_LinkList.size();
				//System.out.println(my_LinkList.size());
				for (int i = 0; i < my_LinkList.size(); i++){
					if (!visited.contains(my_LinkList.get(i)))
					visited.add(my_LinkList.get(i)); //list of links
					my_PageToRetrieveBuffer.offer(my_LinkList.get(i));
					
				}								
			
									
					List<String> PageText = new ArrayList<String>(); //list of text
					try {
						PageText = getText(var_currentLink);
					} catch (IOException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
						
					}
					PageInfo my_PageInfo = new PageInfo( var_currentLink,PageText, totalURL );
					my_PageBuffer.add(my_PageInfo); //add to the PageBuffer
					//my_PageToRetrieveBuffer.add(my_PageInfo);
				   if (my_PageBuffer.size()==50){
					   my_PageToRetrieveBuffer.clear();
				   }
			}
		
	}
	
	private void parsePages(){
		final long totalTime = System.nanoTime(); 
		long parseTime;
		while (my_PageBuffer.isEmpty()==false) {
			parseTime = System.nanoTime();
			PageInfo page  = my_PageBuffer.poll();
			//System.out.println("Reading: " + page.getURL() + "\n");
					
				List<String> tempList= new ArrayList<String>();
				tempList = page.getText();
				Parser parser = new Parser(tempList, my_SeedList);
				Map<String,Integer>tempMap = parser.getAnalytics(tempList);
				my_report.setParseTime(System.nanoTime()-parseTime);
				my_report.addKeyWordCount(tempMap);
				my_report.setCurrentKeyWordCount(tempMap);
				my_report.addTotalPages(1);
				my_report.setNumWords(parser.getTotal());
				my_report.setUrl(page.getURL());
				my_report.addTotalUrls(page.getNumLinks());
				long ttime = System.nanoTime()-startTime;
				my_report.setTotalTime(ttime);
			    my_report.Print();
				//System.out.print(my_report.toString());
			}
	}
	
	private List<String> getText(String the_URL) throws IOException{
		List<String> text = new ArrayList<String>();
		 Document doc = Jsoup.connect(the_URL).get();
	     Elements links = doc.select("a[href]");
	    
	        
	     Elements descriptions = doc.select("p");

	     for (Element element : descriptions) {
	         text.add(element.ownText());
	        
	     }
	     Elements title = doc.select("title");
	     
	     for (Element element : title) {
	    	 text.add(element.ownText());
	        
	     }
		
		return text;
		
	}
	public void Run(){
		my_report = new Report(my_SeedList);
		/**Step 1**/
		//Retriever my_retriever = new Retriever(my_BaseURL);
			//PageInfo firstPage = new PageInfo(my_BaseURL);
			my_PageToRetrieveBuffer.add(my_BaseURL);
			visited.add(my_BaseURL);
			
			/**Step 2 get links and add pages to the Page Buffer**/
			scanPage();
			
			/**Step 3....Parse and display data**/
			parsePages();
			
	
	
	}
	
}
